package com.pig4cloud.pig.common.core;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.StreamProgress;
import cn.hutool.core.lang.Console;
import cn.hutool.core.lang.Dict;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.json.JSONUtil;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * 图片网站图片下载
 */
public class CrawlingImg {
    // 根路径
    public static String rootUrl = "http://pic.netbian.com",
    // 默认下载图片列表
    imgListUrl = rootUrl + "/4kmeinv/",
    // 下载图片列表标题(优先于 默认下载图片列表),如有路径则替换默认下载图片列表路径进行下载;
    title = "",
    // 分页模式
    pageListUrl = imgListUrl + "index_{}.html",
    // 图片信息路径
    imgMsgUrl = rootUrl + "/e/extend/downpic.php",
    // 文件下载路径
    path = "D:\\图片\\",
    // cookie(需页面获取)
    cookie = "__cfduid=d8a8700433d65cc2cd4f55e3080b68cb91590674140; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1590674150,1590926747; PHPSESSID=g9p1qmobf7ark7h1devof2av47; zkhanecookieclassrecord=%2C54%2C; zkhanreturnurl=http%3A%2F%2Fpic.netbian.com%2Fe%2Fmember%2Fbuygroup%2F; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1591018382; zkhanmlusername=qq081992153305; zkhanmluserid=527303; zkhanmlgroupid=3; zkhanmlrnd=XHDGTAHhtr9hxoOw2tIf; zkhanmlauth=0a1d2f57c958dc815f98c3bb5ae396e8";
    // 下载开始页
    public static Integer pageNum = 60,
    // 下载总页数
    totalPage = 66,
    // 下载间隔 (单位秒)，一定设置否则ip会被墙
    sleep = 30,
            index = null;
    public static Elements ip = null;

    public static void main(String[] args) {
        String s = "{transport {\n" +
                "  type = \"TCP\"\n" +
                "  server = \"NIO\"\n" +
                "  heartbeat = true\n" +
                "  enableClientBatchSendRequest = false\n" +
                "  threadFactory {\n" +
                "    bossThreadPrefix = \"NettyBoss\"\n" +
                "    workerThreadPrefix = \"NettyServerNIOWorker\"\n" +
                "    serverExecutorThreadPrefix = \"NettyServerBizHandler\"\n" +
                "    shareBossWorker = false\n" +
                "    clientSelectorThreadPrefix = \"NettyClientSelector\"\n" +
                "    clientSelectorThreadSize = 1\n" +
                "    clientWorkerThreadPrefix = \"NettyClientWorkerThread\"\n" +
                "    bossThreadSize = 1\n" +
                "    workerThreadSize = \"default\"\n" +
                "  }\n" +
                "  shutdown {\n" +
                "    wait = 3\n" +
                "  }\n" +
                "  serialization = \"seata\"\n" +
                "  compressor = \"none\"\n" +
                "}\n" +
                "\n" +
                "store {\n" +
                "  mode = \"file\"\n" +
                "  file {\n" +
                "    dir = \"sessionStore\"\n" +
                "    maxBranchSessionSize = 16384\n" +
                "    maxGlobalSessionSize = 512\n" +
                "    fileWriteBufferCacheSize = 16384\n" +
                "    sessionReloadReadSize = 100\n" +
                "    flushDiskMode = async\n" +
                "  }\n" +
                "\n" +
                "  db {\n" +
                "    datasource = \"druid\"\n" +
                "    dbType = \"mysql\"\n" +
                "    driverClassName = \"com.mysql.jdbc.Driver\"\n" +
                "    url = \"jdbc:mysql://127.0.0.1:3306/seata\"\n" +
                "    user = \"mysql\"\n" +
                "    password = \"mysql\"\n" +
                "    minConn = 5\n" +
                "    maxConn = 30\n" +
                "    globalTable = \"global_table\"\n" +
                "    branchTable = \"branch_table\"\n" +
                "    lockTable = \"lock_table\"\n" +
                "    queryLimit = 100\n" +
                "  }\n" +
                "}\n" +
                "server {\n" +
                "  recovery {\n" +
                "    committingRetryPeriod = 1000\n" +
                "    asynCommittingRetryPeriod = 1000\n" +
                "    rollbackingRetryPeriod = 1000\n" +
                "    timeoutRetryPeriod = 1000\n" +
                "  }\n" +
                "  undo {\n" +
                "    logSaveDays = 7\n" +
                "    logDeletePeriod = 86400000\n" +
                "  }\n" +
                "  maxCommitRetryTimeout = \"-1\"\n" +
                "  maxRollbackRetryTimeout = \"-1\"\n" +
                "  rollbackRetryTimeoutUnlockEnable = false\n" +
                "}\n" +
                "\n" +
                "metrics {\n" +
                "  enabled = false\n" +
                "  registryType = \"compact\"\n" +
                "  exporterList = \"prometheus\"\n" +
                "  exporterPrometheusPort = 9898\n" +
                "}}";
        s= s.replace("=",":").replace("{",":{").replace("}","},");
        System.out.println(s);
        Elements elementRoot = Jsoup.parse(setProxy(HttpRequest.get(rootUrl)).execute().body()).select("body").select("div[class=nav-m clearfix tran]");

        String text = StrUtil.isNotBlank(title) ? rootUrl + elementRoot.get(0).select("a[title=" + title + "]").attr("title") : null;
        imgListUrl = StrUtil.isNotBlank(text) && text.equalsIgnoreCase(title) ? rootUrl + elementRoot.get(0).select("a[title=" + title + "]").attr("href") : imgListUrl;


        List<Elements> elements = new ArrayList();
        List<String> url = new ArrayList();
        if (totalPage.equals(1)) {
            elements.add(createElements(imgListUrl));
        } else {
            for (int i = pageNum; i < totalPage + 1; i++) {
                url.add(StrUtil.replace(pageListUrl, "{}", String.valueOf(i), true));
            }
        }
        if (totalPage.equals(1) && url.size() > 0 && url.size() <= 1) {
            down(elements.get(0));
        } else {
            elements.addAll(url.parallelStream().map(i -> createElements(i)).collect(Collectors.toList()));
            elements.parallelStream().map(i -> {
                try {
                    Thread.sleep(sleep * 1000);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return down(i);
            }).collect(Collectors.toList());
        }
    }

    public static Elements createElements(String url) {
        return Jsoup.parse(setProxy(HttpRequest.get(url).headerMap(createHeaderMap(), true).cookie(cookie)).execute().body()).select("body")
                .select("div[id=main]").select("div[class=slist]").select("ul").select("li");
    }

    public static List down(Elements elements) {
        List<String> list = elements.parallelStream().map(i -> rootUrl + i.select("a").attr("href")).collect(Collectors.toList());
        return list.parallelStream().map(i -> {
            Elements body = Jsoup.parse(setProxy(HttpRequest.get(i).cookie(cookie)).execute().body()).select("body");
            String name = body.select("div[class=photo-hd]").select("h1").text() + ".jpg";
            String result = setProxy(request(imgMsgUrl, cookie, "application/json, text/javascript, */*; q=0.01")
                    .form(
                            Dict.create().set("id", body.select("div[class=downpic]").select("a").attr("data-id"))
                                    .set("t", "0." + RandomUtil.randomString("0123456789", 16))
                    )).execute().body();

            HttpResponse resp = setProxy(request(rootUrl + JSONUtil.toBean(result, Dict.class).getStr("pic"), cookie, "application/json, text/javascript, */*; q=0.01")).execute().sync();
            try {
                Thread.sleep(sleep * 2 * 1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            return resp.writeBody(FileUtil.file(path + name), new StreamProgress() {
                @Override
                public void start() {
//                    Console.log("开始下载 {} 。。。", name);
                }

                @Override
                public void progress(long progressSize) {
                    Console.log("{}...已下载: {}", name, FileUtil.readableFileSize(progressSize));
                }

                @Override
                public void finish() {
                    Console.log("下载完成 {} ... 文件夹文件总数：{} 个", name, FileUtil.ls(path).length);
                }
            });
        }).collect(Collectors.toList());
    }

    private static HttpRequest setProxy(HttpRequest request) {
//        Elements body = Jsoup.parse(HttpRequest.get("https://www.xicidaili.com/nn/").execute().body()).select("body").select("table[id=ip_list]").select("tr");
//        ip = ip == null ? Jsoup.parse(HttpRequest.get("https://www.xicidaili.com/nn/").execute().body()).select("body").select("table[id=ip_list]").select("tr") : ip;
//        index = index == null ? Integer.parseInt(RandomUtil.randomString("123456789",1)) : index;
//        return request.setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ip.get(index).select("td").get(1).text(), Integer.parseInt(ip.get(index).select("td").get(2).text()))));
        return request;
    }

    private static HttpRequest request(String url, String cookie, String contentType) {
        return HttpRequest.get(url).contentType(contentType).cookie(cookie).headerMap(createHeaderMap(), true);
    }


    private static Map createHeaderMap() {
        return Dict.create().set("Host", "pic.netbian.com").set("Connection", "keep-alive").set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
                .set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36");
    }

}